{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FaissEvalTopKColabToGPT2TrainData.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "JcrrSVNi8S5-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#May have a memory crash in CPU or GPU mode. If so, run in TPU mode. \n",
        "\n",
        "#This notebook downloads the FFNN embedding files directly from my google drive \n",
        "\n",
        "#run all cells. The cells at the bottom creates a CSV file"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "odozicZsVrMZ",
        "colab_type": "code",
        "outputId": "0ad10fc6-f147-417c-881c-d60044a9a13d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        }
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import csv\n",
        "from tqdm import tqdm\n",
        "\n",
        "\n",
        "!pip install -U -q PyDrive\n",
        "\n",
        "from google.colab import files\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n",
        "# drive.mount('/content/gdrive')\n",
        "# files = os.listdir(\"/content/gdrive/My Drive/FFNNEmbeds\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K    1% |▎                               | 10kB 19.0MB/s eta 0:00:01\r\u001b[K    2% |▋                               | 20kB 2.3MB/s eta 0:00:01\r\u001b[K    3% |█                               | 30kB 3.4MB/s eta 0:00:01\r\u001b[K    4% |█▎                              | 40kB 2.2MB/s eta 0:00:01\r\u001b[K    5% |█▋                              | 51kB 2.7MB/s eta 0:00:01\r\u001b[K    6% |██                              | 61kB 3.2MB/s eta 0:00:01\r\u001b[K    7% |██▎                             | 71kB 3.6MB/s eta 0:00:01\r\u001b[K    8% |██▋                             | 81kB 4.1MB/s eta 0:00:01\r\u001b[K    9% |███                             | 92kB 4.6MB/s eta 0:00:01\r\u001b[K    10% |███▎                            | 102kB 3.6MB/s eta 0:00:01\r\u001b[K    11% |███▋                            | 112kB 3.6MB/s eta 0:00:01\r\u001b[K    12% |████                            | 122kB 5.0MB/s eta 0:00:01\r\u001b[K    13% |████▎                           | 133kB 5.0MB/s eta 0:00:01\r\u001b[K    14% |████▋                           | 143kB 9.3MB/s eta 0:00:01\r\u001b[K    15% |█████                           | 153kB 9.4MB/s eta 0:00:01\r\u001b[K    16% |█████▎                          | 163kB 9.4MB/s eta 0:00:01\r\u001b[K    17% |█████▋                          | 174kB 9.5MB/s eta 0:00:01\r\u001b[K    18% |██████                          | 184kB 9.5MB/s eta 0:00:01\r\u001b[K    19% |██████▎                         | 194kB 9.6MB/s eta 0:00:01\r\u001b[K    20% |██████▋                         | 204kB 41.8MB/s eta 0:00:01\r\u001b[K    21% |███████                         | 215kB 10.6MB/s eta 0:00:01\r\u001b[K    22% |███████▎                        | 225kB 10.6MB/s eta 0:00:01\r\u001b[K    23% |███████▋                        | 235kB 10.8MB/s eta 0:00:01\r\u001b[K    24% |████████                        | 245kB 10.6MB/s eta 0:00:01\r\u001b[K    25% |████████▎                       | 256kB 10.6MB/s eta 0:00:01\r\u001b[K    26% |████████▋                       | 266kB 10.3MB/s eta 0:00:01\r\u001b[K    27% |█████████                       | 276kB 10.4MB/s eta 0:00:01\r\u001b[K    29% |█████████▎                      | 286kB 10.4MB/s eta 0:00:01\r\u001b[K    30% |█████████▋                      | 296kB 10.4MB/s eta 0:00:01\r\u001b[K    31% |██████████                      | 307kB 10.8MB/s eta 0:00:01\r\u001b[K    32% |██████████▎                     | 317kB 47.6MB/s eta 0:00:01\r\u001b[K    33% |██████████▋                     | 327kB 48.6MB/s eta 0:00:01\r\u001b[K    34% |███████████                     | 337kB 50.0MB/s eta 0:00:01\r\u001b[K    35% |███████████▎                    | 348kB 46.7MB/s eta 0:00:01\r\u001b[K    36% |███████████▋                    | 358kB 47.0MB/s eta 0:00:01\r\u001b[K    37% |████████████                    | 368kB 55.1MB/s eta 0:00:01\r\u001b[K    38% |████████████▎                   | 378kB 55.5MB/s eta 0:00:01\r\u001b[K    39% |████████████▋                   | 389kB 56.0MB/s eta 0:00:01\r\u001b[K    40% |█████████████                   | 399kB 12.5MB/s eta 0:00:01\r\u001b[K    41% |█████████████▎                  | 409kB 12.3MB/s eta 0:00:01\r\u001b[K    42% |█████████████▋                  | 419kB 12.3MB/s eta 0:00:01\r\u001b[K    43% |██████████████                  | 430kB 12.2MB/s eta 0:00:01\r\u001b[K    44% |██████████████▎                 | 440kB 12.2MB/s eta 0:00:01\r\u001b[K    45% |██████████████▋                 | 450kB 12.3MB/s eta 0:00:01\r\u001b[K    46% |███████████████                 | 460kB 12.2MB/s eta 0:00:01\r\u001b[K    47% |███████████████▎                | 471kB 12.2MB/s eta 0:00:01\r\u001b[K    48% |███████████████▋                | 481kB 12.2MB/s eta 0:00:01\r\u001b[K    49% |████████████████                | 491kB 12.2MB/s eta 0:00:01\r\u001b[K    50% |████████████████▎               | 501kB 52.2MB/s eta 0:00:01\r\u001b[K    51% |████████████████▋               | 512kB 50.4MB/s eta 0:00:01\r\u001b[K    52% |█████████████████               | 522kB 50.6MB/s eta 0:00:01\r\u001b[K    53% |█████████████████▎              | 532kB 51.9MB/s eta 0:00:01\r\u001b[K    54% |█████████████████▋              | 542kB 11.1MB/s eta 0:00:01\r\u001b[K    55% |██████████████████              | 552kB 11.1MB/s eta 0:00:01\r\u001b[K    57% |██████████████████▎             | 563kB 11.0MB/s eta 0:00:01\r\u001b[K    58% |██████████████████▋             | 573kB 11.0MB/s eta 0:00:01\r\u001b[K    59% |███████████████████             | 583kB 11.0MB/s eta 0:00:01\r\u001b[K    60% |███████████████████▎            | 593kB 11.0MB/s eta 0:00:01\r\u001b[K    61% |███████████████████▋            | 604kB 11.0MB/s eta 0:00:01\r\u001b[K    62% |████████████████████            | 614kB 11.2MB/s eta 0:00:01\r\u001b[K    63% |████████████████████▎           | 624kB 11.1MB/s eta 0:00:01\r\u001b[K    64% |████████████████████▋           | 634kB 11.1MB/s eta 0:00:01\r\u001b[K    65% |█████████████████████           | 645kB 53.1MB/s eta 0:00:01\r\u001b[K    66% |█████████████████████▎          | 655kB 57.0MB/s eta 0:00:01\r\u001b[K    67% |█████████████████████▋          | 665kB 43.4MB/s eta 0:00:01\r\u001b[K    68% |██████████████████████          | 675kB 43.2MB/s eta 0:00:01\r\u001b[K    69% |██████████████████████▎         | 686kB 43.3MB/s eta 0:00:01\r\u001b[K    70% |██████████████████████▋         | 696kB 43.4MB/s eta 0:00:01\r\u001b[K    71% |███████████████████████         | 706kB 43.3MB/s eta 0:00:01\r\u001b[K    72% |███████████████████████▎        | 716kB 44.2MB/s eta 0:00:01\r\u001b[K    73% |███████████████████████▋        | 727kB 44.7MB/s eta 0:00:01\r\u001b[K    74% |████████████████████████        | 737kB 44.5MB/s eta 0:00:01\r\u001b[K    75% |████████████████████████▎       | 747kB 44.9MB/s eta 0:00:01\r\u001b[K    76% |████████████████████████▋       | 757kB 44.7MB/s eta 0:00:01\r\u001b[K    77% |████████████████████████▉       | 768kB 61.6MB/s eta 0:00:01\r\u001b[K    78% |█████████████████████████▏      | 778kB 62.1MB/s eta 0:00:01\r\u001b[K    79% |█████████████████████████▌      | 788kB 60.9MB/s eta 0:00:01\r\u001b[K    80% |█████████████████████████▉      | 798kB 60.9MB/s eta 0:00:01\r\u001b[K    81% |██████████████████████████▏     | 808kB 60.8MB/s eta 0:00:01\r\u001b[K    82% |██████████████████████████▌     | 819kB 60.0MB/s eta 0:00:01\r\u001b[K    83% |██████████████████████████▉     | 829kB 60.0MB/s eta 0:00:01\r\u001b[K    85% |███████████████████████████▏    | 839kB 60.4MB/s eta 0:00:01\r\u001b[K    86% |███████████████████████████▌    | 849kB 60.6MB/s eta 0:00:01\r\u001b[K    87% |███████████████████████████▉    | 860kB 50.9MB/s eta 0:00:01\r\u001b[K    88% |████████████████████████████▏   | 870kB 50.5MB/s eta 0:00:01\r\u001b[K    89% |████████████████████████████▌   | 880kB 51.5MB/s eta 0:00:01\r\u001b[K    90% |████████████████████████████▉   | 890kB 52.4MB/s eta 0:00:01\r\u001b[K    91% |█████████████████████████████▏  | 901kB 52.4MB/s eta 0:00:01\r\u001b[K    92% |█████████████████████████████▌  | 911kB 52.8MB/s eta 0:00:01\r\u001b[K    93% |█████████████████████████████▉  | 921kB 53.4MB/s eta 0:00:01\r\u001b[K    94% |██████████████████████████████▏ | 931kB 53.8MB/s eta 0:00:01\r\u001b[K    95% |██████████████████████████████▌ | 942kB 53.6MB/s eta 0:00:01\r\u001b[K    96% |██████████████████████████████▉ | 952kB 53.4MB/s eta 0:00:01\r\u001b[K    97% |███████████████████████████████▏| 962kB 66.3MB/s eta 0:00:01\r\u001b[K    98% |███████████████████████████████▌| 972kB 67.6MB/s eta 0:00:01\r\u001b[K    99% |███████████████████████████████▉| 983kB 66.6MB/s eta 0:00:01\r\u001b[K    100% |████████████████████████████████| 993kB 20.5MB/s \n",
            "\u001b[?25h  Building wheel for PyDrive (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h\n",
            "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "dwxg1E7afmZN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import requests"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mmrJD2VIN6fw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def download_file_from_google_drive(id, destination):\n",
        "    URL = \"https://docs.google.com/uc?export=download\"\n",
        "\n",
        "    session = requests.Session()\n",
        "\n",
        "    response = session.get(URL, params = { 'id' : id }, stream = True)\n",
        "    token = get_confirm_token(response)\n",
        "\n",
        "    if token:\n",
        "        params = { 'id' : id, 'confirm' : token }\n",
        "        response = session.get(URL, params = params, stream = True)\n",
        "\n",
        "    save_response_content(response, destination)    \n",
        "\n",
        "def get_confirm_token(response):\n",
        "    for key, value in response.cookies.items():\n",
        "        if key.startswith('download_warning'):\n",
        "            return value\n",
        "\n",
        "    return None\n",
        "\n",
        "def save_response_content(response, destination):\n",
        "    CHUNK_SIZE = 32768\n",
        "\n",
        "    with open(destination, \"wb\") as f:\n",
        "        for chunk in response.iter_content(CHUNK_SIZE):\n",
        "            if chunk: # filter out keep-alive new chunks\n",
        "                f.write(chunk)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "o4kesXGHhIJ9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "if not os.path.exists('filez'):\n",
        "    os.makedirs('filez')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Pi272ybDN98B",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "file_id = '1hbk2z_zwTwKPZ3yKyVjjvSaj5rbnmBmj'\n",
        "\n",
        "download_file_from_google_drive(file_id, 'filez/eHealthForums.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "E9VU-v7mOKb_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "file_id = '1rUU6wrEW72dbC3D7yJB1G5QdUg9UImS6'\n",
        "\n",
        "download_file_from_google_drive(file_id, 'filez/healthTapFFNNEmbeddings.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vhPPNdQgOLk3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "file_id = '1vsLTKh6A7Tzpyf_qHHYgYG4wzX71ThoF'\n",
        "\n",
        "download_file_from_google_drive(file_id, 'filez/askDocsFFNNEmbeddings.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yRUiqpLoOLTg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "file_id = '1y6LzAZDhNwMtnz6QFUwMWKENzayvzeUF'\n",
        "\n",
        "download_file_from_google_drive(file_id, 'filez/webMDFFNNEmbeddings.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ONRtJ9kxOK9p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "file_id = '15x44Pglx3bg46Bqp2HaWR1OW3LFOURXr'\n",
        "\n",
        "download_file_from_google_drive(file_id, 'filez/iClinicFFNNEmbeddings.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "shnSW4yEg7fT",
        "colab_type": "code",
        "outputId": "9cebaf1c-6b27-4524-bf4a-74b25b7e9c25",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "cell_type": "code",
      "source": [
        "os.listdir('filez')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['eHealthForums.csv',\n",
              " 'healthTapFFNNEmbeddings.csv',\n",
              " 'askDocsFFNNEmbeddings.csv',\n",
              " 'webMDFFNNEmbeddings.csv',\n",
              " 'iClinicFFNNEmbeddings.csv']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "metadata": {
        "id": "lvesd_SRN95p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "files = os.listdir('filez')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ftB3GRCJo_dh",
        "colab_type": "code",
        "outputId": "8d932358-04f8-42fb-ecca-1220d2c5c1c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "cell_type": "code",
      "source": [
        "files"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['eHealthForums.csv',\n",
              " 'healthTapFFNNEmbeddings.csv',\n",
              " 'askDocsFFNNEmbeddings.csv',\n",
              " 'webMDFFNNEmbeddings.csv',\n",
              " 'iClinicFFNNEmbeddings.csv']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "metadata": {
        "id": "JW0C5N-UWtwT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def fix_array(x):\n",
        "    x = np.fromstring(\n",
        "    x.replace('\\n','')\n",
        "    .replace('[','')\n",
        "    .replace(']','')\n",
        "    .replace('  ',' '), sep=' ')\n",
        "    return x.reshape((1, 768))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "niyPY7RMz40j",
        "colab_type": "code",
        "outputId": "61ce3e0d-85df-448e-db32-15808ea4092f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "pd.options.display.max_rows = 700\n",
        "pd.set_option('expand_frame_repr', True)\n",
        "pd.set_option('max_colwidth', 250)\n",
        "pd.get_option(\"display.max_rows\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "700"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "metadata": {
        "id": "WqlnLnWnWvqi",
        "colab_type": "code",
        "outputId": "1eaab8ce-8bdd-432a-bb27-a651fc36ea51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "cell_type": "code",
      "source": [
        "qa = pd.read_csv(\"filez/\" + files[0])\n",
        "for file in files[1:]:\n",
        "    print(file)\n",
        "    qa = pd.concat([qa, pd.read_csv(\"filez/\" + file)], axis = 0)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "healthTapFFNNEmbeddings.csv\n",
            "askDocsFFNNEmbeddings.csv\n",
            "webMDFFNNEmbeddings.csv\n",
            "iClinicFFNNEmbeddings.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wGh5zlJs1eOE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "del qa['Unnamed: 0']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hsJKRYwCWyzS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "qa[\"Q_FFNN_embeds\"] = qa[\"Q_FFNN_embeds\"].apply(fix_array)\n",
        "qa[\"A_FFNN_embeds\"] = qa[\"A_FFNN_embeds\"].apply(fix_array)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JoSZ5ImRW1-T",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# qa = qa.sample(frac = 1)\n",
        "# qa.reset_index(inplace = True, drop = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DteXdlBh2jBl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "qa = qa.reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "erQwDeii1Jns",
        "colab_type": "code",
        "outputId": "d9c8701f-9652-4426-a141-82b452e4b4ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 632
        }
      },
      "cell_type": "code",
      "source": [
        "qa.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>Q_FFNN_embeds</th>\n",
              "      <th>A_FFNN_embeds</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>how do i stop smoking now</td>\n",
              "      <td>stopping smoking is about will power and being steadfast. you can stop safely by having bupropion or nicotine patch cover initially in consult with a doctor. contact an addiction clinic near you. wishing you best of health thanks</td>\n",
              "      <td>[[0.198553607, 0.238654211, 0.0252966508, 0.536100566, -0.270931244, 0.143292472, 0.66546452, -0.364487559, 0.831103981, -0.257745147, 1.03932917, 0.0567179397, -0.173779696, 0.332652003, -0.152597591, -0.0945576429, -0.0650781095, 0.41572243, 0....</td>\n",
              "      <td>[[-0.107524604, 0.131869704, -0.135342821, 0.236110285, 0.144869864, 0.125502259, 0.0284669325, -0.241687, 0.359113157, 0.215388954, 0.0614173561, -0.117674321, -0.18703939, 0.422599256, 0.291348606, 0.132292926, -0.0371215381, -0.328066021, 0.38...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i had a tubaligation 4 years ago and also have a minor case of endometriosis i am only 27 and my period stopped completely 3 months ago no more spotting in between nothing i am not stressed no change in daily habits diet weight nothing all urine ...</td>\n",
              "      <td>hello this sounds quite unfamiliar that due to no reason at this age you have no cycle for last three months. pregnancy is surely a remote possibility as there was a tubal ligation. the endometriosis is also unlikely could cause stoppage of cycle...</td>\n",
              "      <td>[[-0.0332615711, 0.0908259526, 0.12959592, 0.188717857, 0.0553386658, -0.138015583, 0.404416293, -0.306202292, -0.27767241, 0.224860445, 1.30278707, 0.0921037346, -0.0639316812, 1.31605291, 0.228724718, -0.043722976, 0.0802280307, 0.422940314, 0....</td>\n",
              "      <td>[[-0.157639518, 0.194728285, 0.0597435087, 0.103870191, 0.252210379, 0.0174597353, 0.201248869, -0.253417432, -0.478795975, 0.379764229, -0.190171927, 0.266052544, 1.04273164, 0.483350039, 0.188350126, 0.873891115, 0.110315099, -0.244599178, 0.17...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>could extra caffeine consumption be a cause of mild unilateral breast tenderness?</td>\n",
              "      <td>extra caffeine can cause gastric discomfort the pain of which can be felt at one side of breast mostly left side but right also is possible and some times whole chest. get evaluated for any local pathology like mastitis. i hope it helps. stay in ...</td>\n",
              "      <td>[[-0.00937359221, 0.218749091, 0.00305952434, 0.443503141, 1.12520206, -0.112999737, 1.00371063, 0.0340681076, 0.292715639, 0.170468584, 1.10793352, 0.0736165494, 0.077357851, -0.298604459, 0.114351489, -0.0838725939, 0.839284778, 1.59145796, 0.2...</td>\n",
              "      <td>[[0.159728408, 0.20174022, 0.047992982, 0.0541549884, 0.538674176, -0.00781832263, -0.068097882, -0.111649357, -0.370063186, 0.439989567, -0.090130806, 0.162251547, -0.00463378336, 0.120588772, 0.182921246, 0.762594581, -0.260490298, -0.395742387...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>hello- i am a 24 year old female 5\"4 &amp; 115 lb &amp; was diagnosed with pcos at 19. i've always been active &amp; extremely healthy. spironolactone &amp; an iud helped with acne &amp; pain but in 2013 i had an ovarian torsion requiring surgery &amp; they saved my rig...</td>\n",
              "      <td>hello thanks for submitting your question here. as per the description you have undergone repeated surgery for ovarian torsion where the ovary is preserved. this sounds a bit different as in torsion it is a dictum to remove the whole ovary. now a...</td>\n",
              "      <td>[[-0.062063057, 0.178399101, 0.0026178658, 0.271553308, 0.306988388, -0.30831778, 0.0854902267, -0.268227369, 0.141994387, 0.482396036, 0.377293408, -0.00115496782, -0.0811023712, 0.627834022, 0.373512119, -0.101564594, -0.0333173573, 0.86133039,...</td>\n",
              "      <td>[[-0.13206917, 0.218641087, -0.142680183, 0.093027018, 0.518154204, -0.0593009144, -0.154638484, -0.39215523, -0.386736423, 0.185574383, -0.0550778024, 0.0291010961, 1.04651797, 0.106718101, 0.237218335, 0.55746001, -0.12029212, -0.188211352, 0.7...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>i was wanting to know if you could tell me if its possible for a woman to go through all their pregnancy without knowing they are pregnant and have periods and ngative pregnancy tests? iwas watching the show called \"i didnt know i was pregnant\" a...</td>\n",
              "      <td>i am glad to help you out. this is not possible scientifically for a woman not to know and still be pregnant throughout the pregnancy. it is also not possible for the periods to continue and still be pregnant. the confirmation can be done by doin...</td>\n",
              "      <td>[[0.00144339097, 0.0378673412, 0.82075876, 0.225200504, 0.207385212, 0.00479039969, 1.20405626, -0.0606160946, 0.276640952, 0.00801353715, 1.08657479, -0.119273856, -0.27343148, 1.38524199, 0.105210721, 0.00458519254, 0.000499391055, -0.272218674...</td>\n",
              "      <td>[[-0.1045039, 0.0604622141, 0.00862282608, -0.0494702943, 0.346262425, -0.184899583, -0.298842371, -0.323502123, -0.447296798, 0.331157982, 0.186825648, -0.0226919744, -0.231269196, 0.375593662, -0.0113884825, -0.0626303554, -0.0791533738, -0.325...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                                                                                                                                                    question  \\\n",
              "0                                                                                                                                                                                                                                  how do i stop smoking now   \n",
              "1  i had a tubaligation 4 years ago and also have a minor case of endometriosis i am only 27 and my period stopped completely 3 months ago no more spotting in between nothing i am not stressed no change in daily habits diet weight nothing all urine ...   \n",
              "2                                                                                                                                                                          could extra caffeine consumption be a cause of mild unilateral breast tenderness?   \n",
              "3  hello- i am a 24 year old female 5\"4 & 115 lb & was diagnosed with pcos at 19. i've always been active & extremely healthy. spironolactone & an iud helped with acne & pain but in 2013 i had an ovarian torsion requiring surgery & they saved my rig...   \n",
              "4  i was wanting to know if you could tell me if its possible for a woman to go through all their pregnancy without knowing they are pregnant and have periods and ngative pregnancy tests? iwas watching the show called \"i didnt know i was pregnant\" a...   \n",
              "\n",
              "                                                                                                                                                                                                                                                      answer  \\\n",
              "0                      stopping smoking is about will power and being steadfast. you can stop safely by having bupropion or nicotine patch cover initially in consult with a doctor. contact an addiction clinic near you. wishing you best of health thanks   \n",
              "1  hello this sounds quite unfamiliar that due to no reason at this age you have no cycle for last three months. pregnancy is surely a remote possibility as there was a tubal ligation. the endometriosis is also unlikely could cause stoppage of cycle...   \n",
              "2  extra caffeine can cause gastric discomfort the pain of which can be felt at one side of breast mostly left side but right also is possible and some times whole chest. get evaluated for any local pathology like mastitis. i hope it helps. stay in ...   \n",
              "3  hello thanks for submitting your question here. as per the description you have undergone repeated surgery for ovarian torsion where the ovary is preserved. this sounds a bit different as in torsion it is a dictum to remove the whole ovary. now a...   \n",
              "4  i am glad to help you out. this is not possible scientifically for a woman not to know and still be pregnant throughout the pregnancy. it is also not possible for the periods to continue and still be pregnant. the confirmation can be done by doin...   \n",
              "\n",
              "                                                                                                                                                                                                                                               Q_FFNN_embeds  \\\n",
              "0  [[0.198553607, 0.238654211, 0.0252966508, 0.536100566, -0.270931244, 0.143292472, 0.66546452, -0.364487559, 0.831103981, -0.257745147, 1.03932917, 0.0567179397, -0.173779696, 0.332652003, -0.152597591, -0.0945576429, -0.0650781095, 0.41572243, 0....   \n",
              "1  [[-0.0332615711, 0.0908259526, 0.12959592, 0.188717857, 0.0553386658, -0.138015583, 0.404416293, -0.306202292, -0.27767241, 0.224860445, 1.30278707, 0.0921037346, -0.0639316812, 1.31605291, 0.228724718, -0.043722976, 0.0802280307, 0.422940314, 0....   \n",
              "2  [[-0.00937359221, 0.218749091, 0.00305952434, 0.443503141, 1.12520206, -0.112999737, 1.00371063, 0.0340681076, 0.292715639, 0.170468584, 1.10793352, 0.0736165494, 0.077357851, -0.298604459, 0.114351489, -0.0838725939, 0.839284778, 1.59145796, 0.2...   \n",
              "3  [[-0.062063057, 0.178399101, 0.0026178658, 0.271553308, 0.306988388, -0.30831778, 0.0854902267, -0.268227369, 0.141994387, 0.482396036, 0.377293408, -0.00115496782, -0.0811023712, 0.627834022, 0.373512119, -0.101564594, -0.0333173573, 0.86133039,...   \n",
              "4  [[0.00144339097, 0.0378673412, 0.82075876, 0.225200504, 0.207385212, 0.00479039969, 1.20405626, -0.0606160946, 0.276640952, 0.00801353715, 1.08657479, -0.119273856, -0.27343148, 1.38524199, 0.105210721, 0.00458519254, 0.000499391055, -0.272218674...   \n",
              "\n",
              "                                                                                                                                                                                                                                               A_FFNN_embeds  \n",
              "0  [[-0.107524604, 0.131869704, -0.135342821, 0.236110285, 0.144869864, 0.125502259, 0.0284669325, -0.241687, 0.359113157, 0.215388954, 0.0614173561, -0.117674321, -0.18703939, 0.422599256, 0.291348606, 0.132292926, -0.0371215381, -0.328066021, 0.38...  \n",
              "1  [[-0.157639518, 0.194728285, 0.0597435087, 0.103870191, 0.252210379, 0.0174597353, 0.201248869, -0.253417432, -0.478795975, 0.379764229, -0.190171927, 0.266052544, 1.04273164, 0.483350039, 0.188350126, 0.873891115, 0.110315099, -0.244599178, 0.17...  \n",
              "2  [[0.159728408, 0.20174022, 0.047992982, 0.0541549884, 0.538674176, -0.00781832263, -0.068097882, -0.111649357, -0.370063186, 0.439989567, -0.090130806, 0.162251547, -0.00463378336, 0.120588772, 0.182921246, 0.762594581, -0.260490298, -0.395742387...  \n",
              "3  [[-0.13206917, 0.218641087, -0.142680183, 0.093027018, 0.518154204, -0.0593009144, -0.154638484, -0.39215523, -0.386736423, 0.185574383, -0.0550778024, 0.0291010961, 1.04651797, 0.106718101, 0.237218335, 0.55746001, -0.12029212, -0.188211352, 0.7...  \n",
              "4  [[-0.1045039, 0.0604622141, 0.00862282608, -0.0494702943, 0.346262425, -0.184899583, -0.298842371, -0.323502123, -0.447296798, 0.331157982, 0.186825648, -0.0226919744, -0.231269196, 0.375593662, -0.0113884825, -0.0626303554, -0.0791533738, -0.325...  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "vnjuTC7LW4Jv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "question_bert = np.concatenate(qa[\"Q_FFNN_embeds\"].values, axis=0)\n",
        "answer_bert = np.concatenate(qa[\"A_FFNN_embeds\"].values, axis=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ODTXaWyDW6SK",
        "colab_type": "code",
        "outputId": "46b40bf9-8e62-4507-8e5c-7df0aff681de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "answer_bert.shape\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(239402, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "metadata": {
        "id": "8_ank7uIW8Nb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "question_bert = question_bert.astype('float32')\n",
        "answer_bert = answer_bert.astype('float32')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6EdFxdx2fGms",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "faiss.normalize_L2(question_bert)\n",
        "faiss.normalize_L2(answer_bert)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Q1DnMFKlx6As",
        "colab_type": "code",
        "outputId": "0cbc0960-c141-4db9-ada5-56789fe6df5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1045
        }
      },
      "cell_type": "code",
      "source": [
        "#just the CPU version of FAISS, will have to look deeper on how to get GPU version, but works fast enough for now \n",
        "\n",
        "!wget  https://anaconda.org/pytorch/faiss-cpu/1.2.1/download/linux-64/faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2\n",
        "!tar xvjf faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2\n",
        "!cp -r lib/python3.6/site-packages/* /usr/local/lib/python3.6/dist-packages/\n",
        "!pip install mkl"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-05-01 12:59:40--  https://anaconda.org/pytorch/faiss-cpu/1.2.1/download/linux-64/faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2\n",
            "Resolving anaconda.org (anaconda.org)... 104.17.92.24, 104.17.93.24, 2606:4700::6811:5c18, ...\n",
            "Connecting to anaconda.org (anaconda.org)|104.17.92.24|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 FOUND\n",
            "Location: https://binstar-cio-packages-prod.s3.amazonaws.com/5a15c9c5c376961204909d87/5aa7f0a65571b411e5c259be?response-content-disposition=attachment%3B%20filename%3D%22faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2%22%3B%20filename%2A%3DUTF-8%27%27faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2&response-content-type=application%2Fx-tar&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=60&X-Amz-Date=20190501T125940Z&X-Amz-SignedHeaders=host&X-Amz-Security-Token=AgoJb3JpZ2luX2VjEKH%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMSJGMEQCIBkt9FKvm8uQXf47qGCNDjttHGN5C5rpeUE%2FekAkiKW7AiA3xCWwarn1ywD2%2BJeG5tvAjlk0lTi%2BzO5DgTdg3WZ8pyrjAwiK%2F%2F%2F%2F%2F%2F%2F%2F%2F%2F8BEAAaDDQ1NTg2NDA5ODM3OCIMl3DP41QdGq75kIvoKrcDqtzCnV7c52ZVrynkYpOrsBz%2FNrvGW5eKhtEbNkXE6BgptQTz9kf4vOA2qJTxaX3c8WTevaGPrTHgKnSwR71VqEuoP4v8zqUCG68Qqr4JXz5IQ%2FjHAjXK7jJwCk1QOgZnsrdQVNkALAD0LcgmrBkfRyPR1OcKIO9hf50d8j5hGamJ3G%2BrSEev%2B747XeO139jdhmzbtf0QhoWYEE0HTwxoX7SnUYQhp6O%2FrlouhHsUN%2F%2F4UHOhMbzr92x7ngx1ew9HhlyZC3n9oenbLMCRaurh6dyq3EHC1StryOzZbqWAWWYVSF2eAHlzcFt0LDTzLo2JKZ8Agv9E%2FbyjZdrSPZLkR2alHhHmN603Y6YKYKBGDVxH6BFSXROXrxZv8KkhuPyMQ5m5ALmZDp%2B9lbgboVxAKp%2B%2FICxNeTOe3Y2SiBcgLlOlKJ%2BipLVj3Q2JJx1kE27YQg3DTVu3ycxdQ5bzj2P6NdftXCGcfip1jRbJz%2FUEfLTKBDYooqjUR5Blm1RcqX1oR7GuEEJi7fVkKoxJWZSMFgkzJprskuM4KhPKiELzKi2R1BUwrpSQe2%2BNLhYYZJSTCpecJaOoPjDMt6XmBTq1AS2bYE8pmad6hGjFaEUmD4WLoQs2xvn7vNQ%2BjK0JzFWuYEIrAvD1VyzugYl3SThPIxWguZ%2FN6O6Zdz%2FxmCX0hxl%2FzK%2FQdiw2UwyJkdsj6RE1UfxvB%2BCshD2DpezMn6oCXdjxBzHww4dVOQj7fasbI2Fkivx9K57Qi5v1oF3bxDMRyRTSJ0JmOgN3DZqXJYV4ZG3JXq3bMN3jX7Iovf1jhoXM4jc%2BDHtiaBRpZYhYOXoPH8tZicU%3D&X-Amz-Credential=ASIAWUI46DZFEAJO4M7K%2F20190501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=f5802ba8054fbf99a14cf4fd1cb2bb88afa65a137feca7e10b6ea61916fe695f [following]\n",
            "--2019-05-01 12:59:40--  https://binstar-cio-packages-prod.s3.amazonaws.com/5a15c9c5c376961204909d87/5aa7f0a65571b411e5c259be?response-content-disposition=attachment%3B%20filename%3D%22faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2%22%3B%20filename%2A%3DUTF-8%27%27faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2&response-content-type=application%2Fx-tar&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=60&X-Amz-Date=20190501T125940Z&X-Amz-SignedHeaders=host&X-Amz-Security-Token=AgoJb3JpZ2luX2VjEKH%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMSJGMEQCIBkt9FKvm8uQXf47qGCNDjttHGN5C5rpeUE%2FekAkiKW7AiA3xCWwarn1ywD2%2BJeG5tvAjlk0lTi%2BzO5DgTdg3WZ8pyrjAwiK%2F%2F%2F%2F%2F%2F%2F%2F%2F%2F8BEAAaDDQ1NTg2NDA5ODM3OCIMl3DP41QdGq75kIvoKrcDqtzCnV7c52ZVrynkYpOrsBz%2FNrvGW5eKhtEbNkXE6BgptQTz9kf4vOA2qJTxaX3c8WTevaGPrTHgKnSwR71VqEuoP4v8zqUCG68Qqr4JXz5IQ%2FjHAjXK7jJwCk1QOgZnsrdQVNkALAD0LcgmrBkfRyPR1OcKIO9hf50d8j5hGamJ3G%2BrSEev%2B747XeO139jdhmzbtf0QhoWYEE0HTwxoX7SnUYQhp6O%2FrlouhHsUN%2F%2F4UHOhMbzr92x7ngx1ew9HhlyZC3n9oenbLMCRaurh6dyq3EHC1StryOzZbqWAWWYVSF2eAHlzcFt0LDTzLo2JKZ8Agv9E%2FbyjZdrSPZLkR2alHhHmN603Y6YKYKBGDVxH6BFSXROXrxZv8KkhuPyMQ5m5ALmZDp%2B9lbgboVxAKp%2B%2FICxNeTOe3Y2SiBcgLlOlKJ%2BipLVj3Q2JJx1kE27YQg3DTVu3ycxdQ5bzj2P6NdftXCGcfip1jRbJz%2FUEfLTKBDYooqjUR5Blm1RcqX1oR7GuEEJi7fVkKoxJWZSMFgkzJprskuM4KhPKiELzKi2R1BUwrpSQe2%2BNLhYYZJSTCpecJaOoPjDMt6XmBTq1AS2bYE8pmad6hGjFaEUmD4WLoQs2xvn7vNQ%2BjK0JzFWuYEIrAvD1VyzugYl3SThPIxWguZ%2FN6O6Zdz%2FxmCX0hxl%2FzK%2FQdiw2UwyJkdsj6RE1UfxvB%2BCshD2DpezMn6oCXdjxBzHww4dVOQj7fasbI2Fkivx9K57Qi5v1oF3bxDMRyRTSJ0JmOgN3DZqXJYV4ZG3JXq3bMN3jX7Iovf1jhoXM4jc%2BDHtiaBRpZYhYOXoPH8tZicU%3D&X-Amz-Credential=ASIAWUI46DZFEAJO4M7K%2F20190501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=f5802ba8054fbf99a14cf4fd1cb2bb88afa65a137feca7e10b6ea61916fe695f\n",
            "Resolving binstar-cio-packages-prod.s3.amazonaws.com (binstar-cio-packages-prod.s3.amazonaws.com)... 52.216.138.67\n",
            "Connecting to binstar-cio-packages-prod.s3.amazonaws.com (binstar-cio-packages-prod.s3.amazonaws.com)|52.216.138.67|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4106816 (3.9M) [application/x-tar]\n",
            "Saving to: ‘faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2’\n",
            "\n",
            "faiss-cpu-1.2.1-py3 100%[===================>]   3.92M  18.7MB/s    in 0.2s    \n",
            "\n",
            "2019-05-01 12:59:41 (18.7 MB/s) - ‘faiss-cpu-1.2.1-py36_cuda9.0.176_1.tar.bz2’ saved [4106816/4106816]\n",
            "\n",
            "info/hash_input.json\n",
            "info/has_prefix\n",
            "info/index.json\n",
            "info/git\n",
            "info/files\n",
            "info/LICENSE.txt\n",
            "info/about.json\n",
            "info/paths.json\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/dependency_links.txt\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/not-zip-safe\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/requires.txt\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/top_level.txt\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/native_libs.txt\n",
            "info/test/run_test.py\n",
            "info/test/run_test.sh\n",
            "info/test/tests/run_tests.sh\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/SOURCES.txt\n",
            "info/recipe/conda_build_config.yaml\n",
            "info/recipe/build.sh\n",
            "info/test/tests/CMakeLists.txt\n",
            "info/test/tests/Makefile\n",
            "info/recipe/meta.yaml.template\n",
            "lib/python3.6/site-packages/faiss-0.1-py3.6.egg-info/PKG-INFO\n",
            "info/test/tests/test_factory.py\n",
            "info/test/tests/test_ivfpq_codec.cpp\n",
            "info/recipe/meta.yaml\n",
            "info/recipe/setup.py\n",
            "info/test/tests/test_blas.cpp\n",
            "info/recipe/makefile.inc\n",
            "info/test/tests/test_ivfpq_indexing.cpp\n",
            "info/test/tests/test_ondisk_ivf.cpp\n",
            "info/test/tests/test_build_blocks.py\n",
            "info/test/tests/test_merge.cpp\n",
            "info/test/tests/test_pairs_decoding.cpp\n",
            "info/test/tests/test_index_composite.py\n",
            "lib/python3.6/site-packages/faiss/__init__.py\n",
            "lib/python3.6/site-packages/faiss/__pycache__/__init__.cpython-36.pyc\n",
            "info/test/tests/test_index.py\n",
            "info/test/tests/test_blas\n",
            "lib/python3.6/site-packages/faiss/__pycache__/swigfaiss.cpython-36.pyc\n",
            "lib/python3.6/site-packages/faiss/swigfaiss.py\n",
            "lib/python3.6/site-packages/faiss/_swigfaiss.so\n",
            "Requirement already satisfied: mkl in /usr/local/lib/python3.6/dist-packages (2019.0)\n",
            "Requirement already satisfied: intel-openmp in /usr/local/lib/python3.6/dist-packages (from mkl) (2019.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oMnnEA8tyA5r",
        "colab_type": "code",
        "outputId": "473d932c-423b-4da8-94de-55d803409fb2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "cell_type": "code",
      "source": [
        "import faiss"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Failed to load GPU Faiss: No module named 'faiss.swigfaiss_gpu'\n",
            "Faiss falling back to CPU-only.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "AO6Tp_IuW-YC",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "answer_index = faiss.IndexFlatIP(answer_bert.shape[-1])\n",
        "answer_index.add(answer_bert)\n",
        "\n",
        "question_index = faiss.IndexFlatIP(question_bert.shape[-1])\n",
        "question_index.add(question_bert)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VISMzNtxvQRy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def topKforGPT2(xf, kf):\n",
        "    D1, I1 = answer_index.search(question_bert[xf:xf+1].astype('float32'), kf)\n",
        "    D2, I2 = question_index.search(question_bert[xf:xf+1].astype('float32'), kf)\n",
        "    return I1, I2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8G5XqKpc30uR",
        "colab_type": "code",
        "outputId": "aa65408b-b0be-4afb-86f4-e0ff63bddb51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "numberSamples=10\n",
        "\n",
        "output = open('GPT2data.csv', \"w\")\n",
        "writer = csv.writer(output)\n",
        "\n",
        "firstrow = ['question', 'answer']\n",
        "for ii in range(0, numberSamples):\n",
        "    firstrow.append('question'+str(ii))\n",
        "    firstrow.append('answer'+str(ii))\n",
        "\n",
        "writer.writerow(firstrow)\n",
        "\n",
        "# for k in tqdm(range(1000), mininterval=30, maxinterval=60):\n",
        "for k in tqdm(range(qa.shape[0]), mininterval=30, maxinterval=60):\n",
        "    rowfill=[]\n",
        "    rowfill.append(qa[\"question\"][k])\n",
        "    rowfill.append(qa[\"answer\"][k])\n",
        "    aa, qq = topKforGPT2(k, int(numberSamples/2))\n",
        "    aaa = qa.loc[list(aa[0]), :]\n",
        "    qqq = qa.loc[list(qq[0]), :]\n",
        "    aaaa =  [*sum(zip( list(aaa['question']) ,list(aaa['answer'])  ),())]\n",
        "    qqqq = [*sum(zip( list(qqq['question']) ,list(qqq['answer'])  ),())]  \n",
        "    finalfill=aaaa+qqqq\n",
        "    rowfill = rowfill + finalfill\n",
        "    writer.writerow(rowfill)\n",
        "    auth.authenticate_user()\n",
        "    gauth = GoogleAuth()\n",
        "    gauth.credentials = GoogleCredentials.get_application_default()\n",
        "    drive = GoogleDrive(gauth)\n",
        "output.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/239402 [00:00<?, ?it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-15766c7cd361>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mrowfill\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqa\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"question\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mrowfill\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqa\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"answer\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0maa\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mqq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtopKforGPT2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnumberSamples\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0maaa\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mqa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maa\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0mqqq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mqa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-54b6cbd0ff45>\u001b[0m in \u001b[0;36mtopKforGPT2\u001b[0;34m(xf, kf)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtopKforGPT2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mD1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mI1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manswer_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion_bert\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mxf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mxf\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float32'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mD2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mI2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquestion_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion_bert\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mxf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mxf\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float32'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mI1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mI2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/faiss/__init__.py\u001b[0m in \u001b[0;36mreplacement_search\u001b[0;34m(self, x, k)\u001b[0m\n\u001b[1;32m    119\u001b[0m         self.search_c(n, swig_ptr(x),\n\u001b[1;32m    120\u001b[0m                       \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswig_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdistances\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m                       swig_ptr(labels))\n\u001b[0m\u001b[1;32m    122\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdistances\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/faiss/swigfaiss.py\u001b[0m in \u001b[0;36msearch\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_swigfaiss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndexFlat_add\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_swigfaiss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndexFlat_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1317\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_swigfaiss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndexFlat_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrange_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_swigfaiss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndexFlat_range_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreconstruct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_swigfaiss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndexFlat_reconstruct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "lvpIWdYM-uGj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "output.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XDwIuhSEfdyF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "478.804"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3IifYduc4Lp6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "uploadModel = drive.CreateFile() \n",
        "uploadModel.SetContentFile('GPT2data.csv')\n",
        "uploadModel.Upload()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}